# coding=utf-8
"""
Function：抓取知乎话题：百万医疗
1、获取源码、评论
2、解析源码
3、stealth.min.js 可隐藏 selenium 的特性、Chrome88+ 需要添加 chrome_options.add_argument("--disable-blink-features=AutomationControlled")
在 driver.get(url)之前执行
"""
import json
import random
import re
import time
import os
import logging

import requests
from selenium import webdriver
from lxml import etree

chrome_options = webdriver.ChromeOptions()
chrome_options.add_experimental_option('excludeSwitches', ['enable-automation'])
chrome_options.add_experimental_option('useAutomationExtension', False)
chrome_options.add_experimental_option('prefs', {'profile_default_content_setting_values': {'images': 2}})
chrome_options.add_argument('--disable-infobars')
chrome_options.add_argument('--start-maximized')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--incognito')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument("--disable-blink-features=AutomationControlled")
chrome_options.add_argument('user-agent={}'.format(
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'))
CHROMEDRIVER_PATH = r"F:\Envs\qt_tools\Scripts\chromedriver.exe"
headers = {
    "User-Agent": 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36'}

driver = webdriver.Chrome(executable_path=CHROMEDRIVER_PATH, options=chrome_options)

f_obj = open("yiliao.json", 'a', encoding="utf-8")

with open('stealth.min.js') as f:
    js = f.read()


def logger(name):
    logger = logging.getLogger(name)
    logger.setLevel(logging.DEBUG)  # 设置日志级别

    # 日志文件名
    log_file_path = "log.txt"

    fh = logging.FileHandler(log_file_path, encoding='utf-8')

    fmt = logging.Formatter("%(asctime)s %(levelname)s %(funcName)s：%(lineno)d  %(pathname)s  %(message)s")  # 日志输出格式
    fh.setFormatter(fmt)
    logger.addHandler(fh)

    return logger


log = logger(__name__)


def execute_js():
    """执行 JS"""
    try:
        driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": js
        })
    except Exception as e:
        log.error(f"执行 js 发生错误：{e}")


def read_file():
    """读取 URL"""
    with open("yiliao_url.txt", encoding='utf-8') as f:
        for line in f:
            if not line:
                continue
            yield line.strip()


def driver_get(url):
    execute_js()  # 执行 js 文件
    log.info(f"正在抓取：{url}")
    driver.get(url)

    driver.switch_to.default_content()
    html = driver.page_source

    # print(html)

    # 获取评论总数目
    comment_url_list = []
    review_count = get_review_total(html)  # 获取评论总数目
    cnt = (review_count // 20) + 1 if isinstance((review_count / 20), float) else (review_count // 20)  # 计算请求次数
    comment_id = get_comment_id(html)  # 获取文章 ID，用于拼接请求获取评论
    for i in range(cnt):
        comment_url = "https://www.zhihu.com/api/v4/articles/{}/root_comments?order=normal&limit=20&offset={}&status=open".format(
            comment_id, i * 20)
        # print(comment_url)
        comment_url_list.append(comment_url)

    # 获取评论列表
    comment_list = get_full_comment(url, comment_url_list)

    data = {
        "url": url,
        "page": str(html),
        "comment_list": comment_list
    }

    save_html(url, data)

    # return html


def get_full_comment(url, comment_url_list):
    """
    获取全部评论
    :param url:
    :param comment_url_list:
    :return:
    """
    res = []
    try:
        for url in comment_url_list:
            r = requests.get(url, headers=headers, timeout=30)
            if r.status_code == 200:
                res.append(r.json())
            time.sleep(random.random())
    except Exception as e:
        log.error(f"{url} 获取评论失败：{e}")

    return res


def get_comment_id(html):
    """获取评论 ID"""
    content = str(html).replace("\n", "").replace("\t", "")
    comment_id = re.findall('"id":(\d+),', content)
    comment_id = comment_id[0] if comment_id else 0
    return comment_id


def get_review_total(html):
    """
    获取评论总数目
    :param html:
    :return:
    """
    review_count = 0
    try:
        obj = etree.HTML(html)
        review_elem = obj.xpath("""//h2[@class="CommentTopbar-title"]/text()""")
        if review_elem:
            is_exists = re.findall("(\d+)", review_elem[0])
            if is_exists:
                review_count = int(is_exists[0])

    except Exception as e:
        log.error(f"获取评论失败：{e}")

    return review_count


def save_html(url, data):
    """
    保存
    :param url:
    :param data:
    :return:
    """
    print(f"{url} 抓取完毕！")
    f_obj.write(json.dumps(data))
    f_obj.write("\n")
    log.info(f"{url} 写入完毕")


def main():
    for line in read_file():
        url = "https:" + line
        driver_get(url)
        time.sleep(10)


if __name__ == '__main__':
    main()
    driver.close()
    driver.quit()


